package fetcher

import (
	"net/http"
	"fmt"
	"golang.org/x/text/transform"
	"io/ioutil"
	"bufio"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"log"
	"time"
	"imooc.com/crawler_distrilbuted/config"
)

//定义一个定时器，每10毫秒执行一下Fetcher
var rateLimiter = time.Tick(config.Qps * time.Millisecond) //10毫秒

func Fetcher(url string) ([]byte, error) {
	<-rateLimiter
	client := http.Client{}
	//创建一个http的request请求
	request, err := http.NewRequest(http.MethodGet,
		url, nil)
	if err != nil {
		return nil, err
	}
	//给request设置头部user-agent
	request.Header.Add("user-agent",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36")
	request.Header.Add("cookie", "sid=c32fd049-0e3a-41f2-9140-b06ca31e1059; ec=dlZLJHWT-1623770212015-69fd9f32d30e51725520308; FSSBBIl1UgzbN7NO=5Y8EUYOm09mMtPMxK6LwX8.xuJcyyZQ2oDKgFS95yYlgQN0ZJ6OnrCbCCqeJBU3tVMXnD7BWziZwrsnWYwkwOuA; Hm_lvt_2c8ad67df9e787ad29dbd54ee608f5d2=1626280553,1626452595,1626534393,1626623054; _exid=ML6NP%2FxYlbcyJj9ia3Qms4iKD%2F%2F5jWYtQWB8cWjH3F8uiQ2o9ojLefyqOAVFxZ%2Bb0rU4IihFQ8ouL81%2BTwUqRA%3D%3D; _efmdata=p3pmkrMCOXgYbRkhgImHbZ%2Fn5RVAAFAVjxkxX7VKZXg3F3u1vdJLtvyKLfcPD5eFZqQcW37LvVvjlwvTfMaBqbCoYIpCJ1E5EeK5SC%2Fq4FY%3D; Hm_lpvt_2c8ad67df9e787ad29dbd54ee608f5d2=1626623082; FSSBBIl1UgzbN7NP=53HpJEKk0s1lqqqm_X9ThRacufC2sZNlvI0je8qWl_JenfqiVClYZbp2aVT5y424ZkOo3G5P.MOnLVgfZSKKFEQdMdgIl8LbLh.9CXFKrOVXgBiTKMZSo_2zgcNlVzrGRAcbd97pogG2lwp_eoDtCBGOXPz0FYnRpls0h.AK9KAeW9_AAL_pY27fp9ohXHPq0MqjnKQ5OoAmpl8udpINTBR7FXTk0fQ8KiJbopORN73_XR.ayBAf8uTwMrjOqRhTXW")
	//使用client请求request请求，并返回response
	resp,err := client.Do(request)
	if err != nil {
		return nil, err
	}
	//关闭response的body输出
	defer resp.Body.Close()
	//判断响应的code是否为200;如果不是报错!
	if resp.StatusCode != http.StatusOK {
		return nil,
		fmt.Errorf("Fetcher error code: %v", resp.StatusCode)
	}
	//检测网页的编码
	bodyReader := bufio.NewReader(resp.Body)
	e := determineEncoding(bodyReader)
	//编码转化,转为UTF-8; bufio.Reader是io.Reader的实现struct
	utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())

	//读取html的代码，不包括head的信息; httputil.download是包括head
	return ioutil.ReadAll(utf8Reader)
}

func determineEncoding(r *bufio.Reader) encoding.Encoding  {
	bytes, err := r.Peek(1024) //peek 1024会存储起来，下次newReader读取的时候就是1025开始读取了
	if err != nil {
		log.Printf("Fetcher error: %v", err)
		return unicode.UTF8
	}
	e, _, _ := charset.DetermineEncoding(bytes, "")

	return e
}
